package com.belonk.spider;

import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.jsoup.Jsoup;
import org.jsoup.internal.StringUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;

/**
 * Created by sun on 2020/4/19.
 *
 * @author sunfuchang03@126.com
 * @version 1.0
 * @since 1.0
 */
public class Starter {
	/*
	 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	 *
	 * Static fields/constants/initializer
	 *
	 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	 */

	private static Logger log = LoggerFactory.getLogger(Starter.class);

	private static final String SUCCESS_SAVE_FILE_NAME = "success";

	private static final String SEPERATOR = "\\|\\|";

	/*
	 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	 *
	 * Instance fields
	 *
	 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	 */

	private final List<String> successList = new ArrayList<>();

	private final ObjectMapper mapper = new ObjectMapper();

	private final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyyMMdd");

	/*
	 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	 *
	 * Constructors
	 *
	 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	 */



	/*
	 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	 *
	 * Methods
	 *
	 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	 */

	public void start() throws IOException {
		String basePath = Starter.class.getResource("/").getPath();
		String domain   = "https://javaweb.shop";

		log.info("Start to crawl site : " + domain);

		// 准备工作
		prepare(basePath);

		int totalPage = 0;
		int startPage = 0;
		int pageSize  = 30;
		while (true) {
			log.info("Starting to parse html at page " + startPage + "...");

			String url = "https://javaweb.shop/article/page?pageNo=" + (++startPage) + "&pageSize=" + pageSize + "&sortType=time";

			// 解析文章列表
			List<Item> items = queryList(url);
			// 处理文章信息
			for (Item item : items) {
				String category  = item.getCategoryName();
				String title     = item.getArticleTitle();
				String detailUrl = domain + "/article/" + item.getArticleId() + ".html";
				String desc      = item.getBriefContent();

				log.info("Processing article : " + title + "...");

				if (successList.contains(title)) {
					log.warn("Article have been processed, skip it: " + title);
					continue;
				}

				Article article = new Article();
				article.setTitle(title);
				article.setCategory(category);
				article.setDesc(desc);

				Document detailDoc        = Jsoup.connect(detailUrl).get();
				Elements articleContentEl = detailDoc.select("article.article-content");

				// 解析下载链接
				if (!parseDownloadInfo(article, detailDoc, articleContentEl)) {
					continue;
				}

				// 创建图片目录，下载图片
				downloadImages(basePath, domain, article, articleContentEl);

				// 存储文件
				writeFile(basePath, item, article);

				// 写入成功列表
				writeSuccessFile(basePath, article);
			}
			log.info("Parse html success at page " + startPage);

			totalPage++;
			if (pageSize > items.size()) {
				break;
			}
		}
		log.info("Crawl site finished : " + domain + ", total page size : " + totalPage);
	}

	private void writeSuccessFile(String basePath, Article article) throws IOException {
		File        successFile = new File(basePath + "/" + SUCCESS_SAVE_FILE_NAME);
		FileWriter  writer      = new FileWriter(successFile, true);
		PrintWriter printWriter = new PrintWriter(writer);
		printWriter.println(article.getTitle() + "||" + article.getDownloadUrl() + "," + article.getDownloadPassword());
		writer.flush();
		printWriter.flush();
		writer.close();
		printWriter.close();
		successList.add(article.getTitle());
	}

	private boolean parseDownloadInfo(Article article, Document detailDoc, Elements articleContentEl) {
		String downloadUrl = null;
		String downLoadPwd = null;
		downloadUrl = articleContentEl.select("div.js-download-container > p.down-tip > span").text();
		if (downloadUrl.contains("隐藏的下载链接")) {
			Elements downLoadEl = detailDoc.select("p.js-download-info");
			downloadUrl = downLoadEl.select("a").text();
			downLoadPwd = downLoadEl.select("span.down-pwd").text();
		} else {
			log.error("Unknown download url：" + downloadUrl + ", Article infomation : " + article);
			return false;
		}
		article.setDownloadUrl(downloadUrl);
		article.setDownloadPassword(downLoadPwd);
		return true;
	}

	private void writeFile(String basePath, Item item, Article article) throws IOException {
		String category    = article.getCategory();
		String title       = article.getTitle();
		String desc        = article.getDesc();
		String downloadUrl = article.getDownloadUrl();
		String downLoadPwd = article.getDownloadPassword();
		log.info("    -> Saving article file : " + category + "/" + title + "...");
		File categoryFile = new File(basePath + "/" + category);
		if (!categoryFile.exists()) {
			boolean b = categoryFile.mkdir();
			assert b;
		}
		File articleFile = new File(basePath + "/" + category + "/" + item.getArticleId());
		if (articleFile.exists()) {
			log.warn("    -> Saving article file exists, skip it ...");
			// boolean b = articleFile.delete();
			// assert b;
		} else {
			boolean b = articleFile.createNewFile();
			assert b;
			FileWriter  writer      = new FileWriter(articleFile);
			PrintWriter printWriter = new PrintWriter(writer);
			printWriter.println(title);
			printWriter.println(category);
			printWriter.println("下载链接：" + downloadUrl + " 密码：" + downLoadPwd);
			printWriter.println(desc);
			printWriter.println();
			printWriter.println(article.getDetail());
			writer.flush();
			printWriter.flush();
			writer.close();
			printWriter.close();
		}
		log.info("Process article success");
	}

	private void downloadImages(String basePath, String domain, Article article, Elements articleContentEl) throws IOException {
		Elements articleDetailEl = articleContentEl.select("div.article-detail");
		Elements imgEls          = articleDetailEl.select("img");
		if (imgEls.size() > 0) {
			log.info("    -> Downloading images, count : " + imgEls.size() + "...");
		}
		List<String> attas = new ArrayList<>();
		article.setAttachmentPath(attas);
		for (Element imgEl : imgEls) {
			String src = imgEl.attr("src");
			if (!StringUtil.isBlank(src)) {
				attas.add(src);

				String imgUrl = null;
				if (src.startsWith("http")) {
					src = src.replace("https://www.javaweb.shop", "");
					src = src.replace("https://javaweb.shop", "");
					if (src.startsWith("http")) {
						// 重新上传
						imgUrl = src;
						src = "/upload/image/" + dateTimeFormatter.format(LocalDateTime.now()) + "/" + src.substring(src.lastIndexOf("/") + 1);
						imgEl.attr("src", src);
					} else {
						imgUrl = domain + src;
					}
				} else {
					imgUrl = domain + src;
				}
				String imgFullPath = basePath + src;
				File   img         = new File(imgFullPath);
				if (img.exists()) {
					log.warn("    -> Image exists, skip it: " + imgFullPath);
					continue;
				}

				String path    = src.substring(0, src.lastIndexOf("/"));
				File   saveDir = new File(basePath + path);
				if (!saveDir.exists()) {
					boolean mkdirs = saveDir.mkdirs();
					assert mkdirs;
				}

				// Save image
				log.info("    -> saving image : " + imgUrl);
				URLConnection uri = new URL(imgUrl).openConnection();
				try (InputStream is = uri.getInputStream()) {
					byte[]               buff                 = new byte[1024];
					FileOutputStream     fos                  = new FileOutputStream(img);
					BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fos);
					int                  length;
					while ((length = is.read(buff)) != -1) {
						bufferedOutputStream.write(buff, 0, length);
					}
					bufferedOutputStream.flush();
				} catch (FileNotFoundException e) {
					log.error("Image file not found, skip it");
				}
			}
		}
		// 可能更新了图片地址
		String detailContent = articleDetailEl.html();
		article.setDetail(detailContent);
	}

	private List<Item> queryList(String url) throws IOException {
		URLConnection  pageUri = new URL(url).openConnection();
		InputStream    stream  = pageUri.getInputStream();
		BufferedReader reader  = new BufferedReader(new InputStreamReader(stream));
		String         line;
		StringBuilder  sb      = new StringBuilder();
		while ((line = reader.readLine()) != null) {
			sb.append(line);
		}
		String json = sb.toString();
		return mapper.readValue(json, new TypeReference<List<Item>>() {
		});
	}

	private void prepare(String basePath) throws IOException {
		log.info("Parsing finished articles...");
		File successFile = new File(basePath + "/" + SUCCESS_SAVE_FILE_NAME);
		if (successFile.exists()) {
			FileReader     successFileReader = new FileReader(successFile);
			BufferedReader bufferedReader    = new BufferedReader(successFileReader);
			while (true) {
				String readLine = bufferedReader.readLine();
				if (StringUtil.isBlank(readLine)) {
					break;
				}
				successList.add(readLine.split(SEPERATOR)[0]);
			}
		}
	}

	public static void main(String[] args) throws IOException {
		new Starter().start();
	}
}